
                /*********************************************
                **                                          **
                **                                          **
                **                 sa.poll.C                **
                **                                          **
                **                                          **
                *********************************************/
                /*********************************************
                **                                          **
                **         Author Jean-Marc Adamo           **
                **       Copyright (C) 1997-2008 JMA        **
                *********************************************/


/*
Spread Arrays and 
      Non-Blocking Global Read/Write Functions
This program is intended for execution with 3 processess. 
Each processor i performs the three following steps:

    - First step:
        - contribute to SpreadArrays SA0, SA1, SA2 creation,
        - initialize SAi local section,
        - subscribe to global synchronization.
    -Second step:
        - write 5i everywhere in SAi,
        - wait for completion of all the write calls (wait_while),
        - wait for global completion of all the write calls (Sdf::barrier).
    -Third step:
        - read any element of SA(i+2)%3 and store the element in 
              local array LAi,
        - wait for completion of all the read calls (wait_while),
            - wait for global completion of all the read calls (Sdf::barrier).

This program could be written with blocking global read/write functions 
as well (see sample program 7). Polling is explicitly performed via the 
wait_while macro. This could be left to the library (see sample program 6).  
Also, the assignment operator could be used instead of global read/write 
functions. 
*/

#include "arch.h"

Global_id gid1;
Global_id gid2;
Global_id gid3;
Coll_id cid0, cid1, cid2, cid3;

void init_gid(int, char **){
   gid1.set();
   gid2.set();
   gid3.set();
   cid0.set();
   cid1.set();
   cid2.set();
   cid3.set();
   fprintf(Sdf::trace, "from node %d: gid1.tag=%d, gid2.tag=%d, gid3.tag=%d, cid0.tag=%d, cid1.tag=%d, cid2.tag=%d, cid3.tag=%d\n",
               Sdf::PROC_ID, gid1.get(), gid2.get(), gid3.get(), cid0.get(), cid1.get(), cid2.get(), cid3.get());
   fflush(Sdf::trace);
}

#define d0 4
#define d1 5
#define d2 2

class RW_count{
   public:
      int cnt;

      RW_count(){
         cnt = 0;
      }
      void reset(){
         cnt = 0;
      }
      void more_RW(int i){
         cnt += i;
      }
};

#define wait_while(condition) while(condition) Sdf::poll();

#define RW_not_complete() (count->cnt < d0*d1*d2)

RW_count *count;

void upon_end_of_RW(HRW_rcd *){
   count->cnt++;
}

class Root_thread: public Thread{
   public:
        Root_thread()
                : Thread((Tmf0_ptr)&Root_thread::body){}

        void body(){
           fprintf(Sdf::trace, "\n\n\nmain() is started\n\n\n");
           fflush(Sdf::trace);
           int (*LA0)[d1][d2];
           int (*LA1)[d1][d2];
           int (*LA2)[d1][d2];

           if(!Sdf::PROC_ID)
              LA0 =  new int[d0][d1][d2];
           else
              if(Sdf::PROC_ID == 1)
                 LA1 = new int[d0][d1][d2];
              else
                 LA2 = new int[d0][d1][d2];

           SpreadArray<int> SA0(gid1, d0, d1, d2, 2);
           SpreadArray<int> SA1(gid2, d0, d1, d2, 2);
           SpreadArray<int> SA2(gid3, d0, d1, d2, 2);

           int *local_ptr0 = SA0.to_local_ptr();
           int *local_ptr1 = SA1.to_local_ptr();
           int *local_ptr2 = SA2.to_local_ptr();

           int d = (d0*d1*d2)/Sdf::PROC_NB + 1;

           for(int i = 0; i < d; i++){
              local_ptr0[i] = 1000; //to check value change
              local_ptr1[i] = 1001; //to check value change
              local_ptr2[i] = 1002; //to check value change
           }

           Sdf::barrier(cid0);

           RW_count c;
           count = &c;

           RWCtrl ctrl(upon_end_of_RW, 0);
           SA0.set_ctrl_ptr(&ctrl);
           SA1.set_ctrl_ptr(&ctrl);
           SA2.set_ctrl_ptr(&ctrl);

           if(!Sdf::PROC_ID){
              int i ;
              for(i = 0; i < d0; i++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       SA0(i, j, k).write(50);
                    }
              wait_while(RW_not_complete());
              Sdf::barrier(cid1);
              count->reset();
              for(i = 0; i < d0; i++)
                 for(int j = 0; j < d1; j++)
                    for(int k = 0; k < d2; k++){
                       SA2(i, j, k).read(LA0[i][j][k]);
                    }
              wait_while(RW_not_complete());
              Sdf::barrier(cid2);
           }else{
              if(Sdf::PROC_ID == 1){
                 int i ;
                 for(i = 0; i < d0; i++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA1(i, j, k).write(51);
                       }
                 wait_while(RW_not_complete());
                 Sdf::barrier(cid1);
                 count->reset();
                 for(i = 0; i < d0; i++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA0(i, j, k).read(LA1[i][j][k]);
                       }
                 wait_while(RW_not_complete());
                 Sdf::barrier(cid2);
              }else{ //Sdf::PROC_ID == 2
                 int i ;
                 for(i = 0; i < d0; i++)
                    for(int j = 0; j < d1; j++)
                       for(int k = 0; k < d2; k++){
                          SA2(i, j, k).write(52);
                       }
                wait_while(RW_not_complete());
                Sdf::barrier(cid1);

                count->reset();
                for(i = 0; i < d0; i++)
                   for(int j = 0; j < d1; j++)
                      for(int k = 0; k < d2; k++){
                         SA1(i, j, k).read(LA2[i][j][k]);
                      }
                wait_while(RW_not_complete());
                Sdf::barrier(cid2);
              }
           }
           for(int i1 = 0; i1 < d; i1++){
              fprintf(Sdf::trace, "node%d:local_ptr0=%d,local_ptr0[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr0, i1,  local_ptr0[i1]);
              fprintf(Sdf::trace, "node%d:local_ptr1=%d,local_ptr1[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr1, i1,  local_ptr1[i1]);
              fprintf(Sdf::trace, "node%d:local_ptr2=%d,local_ptr2[%d]=%d\n",Sdf::PROC_ID,
                           local_ptr2, i1,  local_ptr2[i1]);
              fflush(Sdf::trace);
           }
           for(int i2 = 0; i2 < d0; i2++)
             for(int j = 0; j < d1; j++)
                for(int k = 0; k < d2; k++){
                   if(Sdf::PROC_ID == 0)
                      fprintf(Sdf::trace, "node%d:  LA0[%d][%d][%d] = %d\n", Sdf::PROC_ID, 
                                   i2, j, k,  LA0[i2][j][k]);
                   if(Sdf::PROC_ID == 1)
                      fprintf(Sdf::trace, "node%d:  LA1[%d][%d][%d] = %d\n", Sdf::PROC_ID,
                                  i2, j, k,  LA1[i2][j][k]);
                   if(Sdf::PROC_ID == 2)
                      fprintf(Sdf::trace, "node%d:  LA2[%d][%d][%d] = %d\n", Sdf::PROC_ID,
                                  i2, j, k,  LA2[i2][j][k]);
                   fflush(Sdf::trace);
                }
           if(!Sdf::PROC_ID)
              delete []LA0;
           else
              if(Sdf::PROC_ID == 1)
                 delete []LA1;
              else
                 delete []LA2;
           fprintf(Sdf::trace, "node%d: the end\n\n\n", Sdf::PROC_ID);
           fflush(Sdf::trace);

           Sdf::barrier(cid3);
        }
};

MAIN(Root_thread, init_gid, Sdf::empty, Sdf::empty)
